"""
Copyright (c) 2022 Huawei Technologies Co.,Ltd.

openGauss is licensed under Mulan PSL v2.
You can use this software according to the terms and conditions of the Mulan PSL v2.
You may obtain a copy of Mulan PSL v2 at:

          http://license.coscl.org.cn/MulanPSL2

THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
See the Mulan PSL v2 for more details.
"""
"""
Case Type   : 故障&可靠性测试
Case Name   : 备节点备份过程中failover
Create At   : 2021/01/04
@zou_jialiang0505328126
Description :
    1.停止主节点
    2.备节点执行备份指令
    3.备份过程中failover
    4.使用备份文件恢复数据库
    5.重启集群
    6.若备节点状态异常对备库进行build
    7.查询数据是否与备份时一致
    8.新主节点执行备份指令
    9使用备份文件恢复数据库
    10.重启集群
    11.若备节点状态异常对备库进行build
    12.查询数据是否与备份时一致
Expect      :
    1.停止主节点成功
    2.备节点备份成功或失败，提示节点已切换
    3.备升主成功
    4.恢复数据库成功
    5.重启集群成功，可能提示集群状态为degrade
    6.重建成功
    7.主备一致
    8.备份成功
    9.恢复成功
    10.重启集群成功，可能提示集群状态为degrade
    11.build成功
    12主备一致
History     :
    modified: 2021/01/08  modified by @wan005,failover过程中备份可能失败，用例中增加该情况
    modified: 2021/01/27  modified by @wan005,failover该场景下可能无法生成备份文件，删除断言
    modified by @peilinqian 2021/2/3: 修改备份路径，解决因原路径为home路径导致备份磁盘不足情况
    modified by @wan005: 2021/05/17 ,调整wal_sender_timeout,规避因环境性能差导致的失败
    modified by @wan005: 2021/06/22 ,备份被中断 增加失败信息匹配,增加switchover前执行build
    modified by @wan005: 2021/07/02 ,优化备份结果判断
"""

import os
import unittest
import time

from yat.test import macro
from yat.test import Node
from testcase.utils.CommonSH import CommonSH
from testcase.utils.CommonSH import RestartDbCluster
from testcase.utils.Common import Common
from testcase.utils.Constant import Constant
from testcase.utils.Logger import Logger
from testcase.utils.ComThread import ComThread

Primary_SH = CommonSH('PrimaryDbUser')


@unittest.skipIf(1 == Primary_SH.get_node_num(), '单机环境不执行')
class Basebackup(unittest.TestCase):
    nodes_tuple = ('PrimaryDbUser', 'Standby1DbUser', 'Standby2DbUser')

    @RestartDbCluster(*nodes_tuple)
    def setUp(self):
        self.log = Logger()
        self.log.info(
            '----Opengauss_Reliability_Tools_Basebackup_Case037.py 开始执行----')
        self.pri_dbuser = Node(node='PrimaryDbUser')
        self.sta1_dbuser = Node(node='Standby1DbUser')
        self.sta2_dbuser = Node(node='Standby2DbUser')
        self.sta1_root = Node(node='Standby1Root')
        self.pri_root = Node(node='PrimaryRoot')
        self.pri_sh = CommonSH('PrimaryDbUser')
        self.sta_1_sh = CommonSH('Standby1DbUser')
        self.sta_2_sh = CommonSH('Standby2DbUser')
        self.Constant = Constant()
        self.common = Common('PrimaryRoot')
        self.parent_path = os.path.dirname(macro.DB_INSTANCE_PATH)
        self.backup_path = os.path.join(self.parent_path, 'base_backup')

        status = self.pri_sh.getDbClusterStatus('detail')
        self.log.info(status)
        self.assertTrue("Normal" in status or "Degraded" in status)

        result = self.pri_sh.executDbSql('show wal_sender_timeout;')
        self.log.info(f"wal_sender_timeout is {result}")
        self.wal_sender_timeout = result.strip().splitlines()[-2]

    def test_basebackup(self):
        self.log.info('-----------修改wal_sender_timeout------------------')
        result = self.pri_sh.executeGsguc('reload',
                                          self.Constant.GSGUC_SUCCESS_MSG,
                                          'wal_sender_timeout=12s')
        self.assertTrue(result)

        self.log.info('----创建备份路径----')
        shell_cmd = f'rm -rf {self.backup_path}; mkdir {self.backup_path}'
        self.log.info(shell_cmd)
        result = self.sta1_dbuser.sh(shell_cmd).result()
        self.log.info(result)

        self.log.info('----1.停止主节点----')
        result = self.pri_sh.stopDbInstance()
        self.assertIn(self.Constant.GS_CTL_STOP_SUCCESS_MSG, result)

        self.log.info('----2.备节点执行备份指令----')
        basebackup_thread = ComThread(self.sta_1_sh.execu_gs_basebackup,
                                      args=(self.backup_path,
                                            self.sta1_dbuser.db_host,
                                            self.sta1_dbuser.db_port,))
        basebackup_thread.setDaemon(True)
        basebackup_thread.start()

        self.log.info('----3.备份过程中failover----')
        failover_success_msg = self.Constant.FAILOVER_SUCCESS_MSG
        result = self.sta_1_sh.executeGsctl('failover',
                                            failover_success_msg,
                                            '')
        self.assertTrue(result)
        result = self.sta_1_sh.exec_refresh_conf()
        self.assertTrue(result)

        self.log.info('====查看备份结果====')
        basebackup_thread.join(60 * 60)
        result = basebackup_thread.get_result()
        self.log.info(result)
        result_num = self.sta1_dbuser.sh(
            f'ls -l {self.backup_path} | grep "^-" | wc -l').result()
        self.log.info(result_num)
        self.assertTrue(result_num.isdigit())
        time.sleep(2)

        if result is None or 'error' in result or 'FATAL' in result:
            self.log.info('主备节点已切换，备份被中断')
            result = self.pri_sh.startDbInstance('standby')
            self.assertIn('server started ', result)
        else:
            self.log.info('----4.使用备份文件恢复数据库----')
            self.assertGreater(int(result_num), 20)
            result = self.sta_1_sh.stopDbCluster()
            self.assertTrue(result)
            shell_cmd = f'rm -rf  {macro.DB_INSTANCE_PATH}/* ;' \
                f'cp -R {self.backup_path}/* {macro.DB_INSTANCE_PATH}'
            self.log.info(shell_cmd)
            result = self.sta1_dbuser.sh(shell_cmd).result()
            self.log.info(result)
            time.sleep(2)

        self.log.info('----5.重启集群，启动数据库成功或出现因为备机为Need repair状态的失败----')
        result = self.pri_sh.startDbCluster('get_detail')
        self.log.info(result)
        result = ('Degraded' in result) or (
                self.Constant.START_SUCCESS_MSG in result)
        self.assertTrue(result)
        time.sleep(2)

        self.log.info('----6.若备节点状态异常对备库进行build----')
        result = self.pri_sh.check_whether_need_build()
        build_success_msg = self.Constant.BUILD_SUCCESS_MSG
        if result:
            result = self.pri_sh.executeGsctl('build',
                                              build_success_msg,
                                              '-b full')
            self.assertTrue((result))
            result = self.sta_2_sh.executeGsctl('build',
                                                build_success_msg,
                                                '-b full')
            self.assertTrue((result))

        self.log.info('----检查主备是否同步----')
        result = self.sta_1_sh.check_data_consistency()
        self.assertTrue(result)
        result = self.sta_2_sh.check_data_consistency()
        self.assertTrue(result)

        self.log.info('----7.查询数据是否与备份时一致----')
        nodes_tuple = (self.pri_dbuser, self.sta1_dbuser, self.sta2_dbuser)
        flag = self.common.check_data_sample_by_all(r'\d', *nodes_tuple)
        self.assertTrue(flag)

        table_dict = self.common.format_sql_result(
            self.pri_sh.executDbSql(r'\d'))
        table_name = table_dict.get('Name')
        for name in table_name:
            select_sql = f'select count(*) from {name};'
            flag = self.common.check_data_sample_by_all(select_sql,
                                                        *nodes_tuple)
            self.assertTrue(flag)

        self.log.info('----8.新主节点执行备份指令----')
        result = self.sta1_dbuser.sh(
            f"rm -rf {self.backup_path}; mkdir {self.backup_path}").result()
        self.log.info(result)
        result = self.sta_1_sh.execu_gs_basebackup(self.backup_path,
                                                   self.sta1_dbuser.db_host,
                                                   self.sta1_dbuser.db_port)
        self.assertNotIn(self.Constant.SQL_WRONG_MSG[1], result)
        self.assertNotIn('error', result)
        self.assertIn(self.Constant.BASEBACKUP_SUCCESS_MSG, result)
        result = self.sta1_dbuser.sh(
            f'ls -l {self.backup_path} | grep "^-" | wc -l').result()
        self.log.info(result)
        self.assertLess(0, int(result))
        time.sleep(2)

        self.log.info('----9.使用备份文件恢复数据库----')
        result = self.sta_1_sh.stopDbCluster()
        self.assertTrue(result)
        shell_cmd = f'rm -rf  {macro.DB_INSTANCE_PATH}/* ;' \
            f'cp -R {self.backup_path}/* {macro.DB_INSTANCE_PATH}'
        self.log.info(shell_cmd)
        result = self.sta1_dbuser.sh(shell_cmd).result()
        self.log.info(result)
        time.sleep(2)

        self.log.info('----10.重启集群，启动数据库成功或出现因为备机为Need repair状态的失败----')
        result = self.pri_sh.startDbCluster('get_detail')
        self.log.info(result)
        result = ('Degraded' in result) or (
                self.Constant.START_SUCCESS_MSG in result)
        self.assertTrue(result)
        time.sleep(2)

        self.log.info('----11.若备节点状态异常对备库进行build----')
        result = self.pri_sh.check_whether_need_build()
        if result:
            result = self.pri_sh.executeGsctl('build',
                                              build_success_msg,
                                              '-b full')
            self.assertTrue((result))
            result = self.sta_2_sh.executeGsctl('build',
                                                build_success_msg,
                                                '-b full')
            self.assertTrue((result))

        self.log.info('----检查主备是否同步----')
        result = self.sta_1_sh.check_data_consistency()
        self.assertTrue(result)
        result = self.sta_2_sh.check_data_consistency()
        self.assertTrue(result)

        self.log.info('----12.查询数据是否与备份时一致----')
        nodes_tuple = (self.pri_dbuser, self.sta1_dbuser, self.sta2_dbuser)
        flag = self.common.check_data_sample_by_all(r'\d', *nodes_tuple)
        self.assertTrue(flag)

        table_dict = self.common.format_sql_result(
            self.pri_sh.executDbSql(r'\d'))
        table_name = table_dict.get('Name')
        for name in table_name:
            select_sql = f'select count(*) from {name};'
            flag = self.common.check_data_sample_by_all(select_sql,
                                                        *nodes_tuple)
            self.assertTrue(flag)

    def tearDown(self):
        self.log.info('----环境清理----')
        self.log.info('----检查是否需要主备切换----')
        status = self.pri_sh.getDbClusterStatus('detail')
        self.log.info(status)
        hostname = self.pri_root.sh("hostname").result()
        result = self.pri_sh.check_whether_need_switch(hostname)
        if result:
            result = self.pri_sh.startDbInstance('standby')
            self.log.info(result)
            status = self.pri_sh.getDbClusterStatus('detail')
            self.log.info(status)
            result = self.pri_sh.check_whether_need_build()
            build_success_msg = self.Constant.BUILD_SUCCESS_MSG
            if result:
                result = self.pri_sh.executeGsctl('build',
                                                  build_success_msg,
                                                  '-b full')
                self.assertTrue((result))
                result = self.sta_2_sh.executeGsctl('build',
                                                    build_success_msg,
                                                    '-b full')
                self.assertTrue((result))
            status = self.pri_sh.getDbClusterStatus('detail')
            self.log.info(status)
            self.pri_sh.executeGsctl('switchover',
                                     self.Constant.SWITCH_SUCCESS_MSG, '')
            self.sta_1_sh.exec_refresh_conf()

        status = self.pri_sh.getDbClusterStatus('detail')
        self.log.info(status)
        self.sta_1_sh.stopDbCluster()
        status = self.pri_sh.getDbClusterStatus('detail')
        self.log.info(status)
        result = self.pri_sh.startDbCluster('get_detail')
        self.log.info(result)

        self.log.info('----删除备份文件----')
        shell_cmd = f'rm -rf {self.backup_path}'
        self.log.info(shell_cmd)
        result = self.sta1_dbuser.sh(shell_cmd).result()
        self.log.info(result)

        self.log.info('-----------还原wal_sender_timeout------------------')
        result = self.pri_sh.executeGsguc('reload',
                                          self.Constant.GSGUC_SUCCESS_MSG,
                                          f'wal_sender_timeout='
                                          f'{self.wal_sender_timeout}')
        self.log.info(result)
        self.log.info(
            '----Opengauss_Reliability_Tools_Basebackup_Case037.py 执行结束----')
